<!DOCTYPE html>
<html>
<head><meta name="generator" content="Hexo 3.9.0">
  <meta charset="utf-8">
  
  <title>月迹的博客</title>
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
  <meta name="description" content="月迹的博客，记录工作或生活中的学习历程，欢迎大家共同讨论！">
<meta name="keywords" content="yueji,月迹,博客,月迹的博客,大数据,Hadoop,HBase,学习,笔记">
<meta property="og:type" content="website">
<meta property="og:title" content="月迹的博客">
<meta property="og:url" content="https://xyueji.github.io/index.html">
<meta property="og:site_name" content="月迹的博客">
<meta property="og:description" content="月迹的博客，记录工作或生活中的学习历程，欢迎大家共同讨论！">
<meta property="og:locale" content="zh-CN">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="月迹的博客">
<meta name="twitter:description" content="月迹的博客，记录工作或生活中的学习历程，欢迎大家共同讨论！">
  
    <link rel="alternative" href="/atom.xml" title="月迹的博客" type="application/atom+xml">
  
  
    <link rel="icon" href="/img/favicon.png">
  
  
      <link rel="stylesheet" href="//cdn.bootcss.com/animate.css/3.5.0/animate.min.css">
  
  <link rel="stylesheet" href="css/style.css">
  <link rel="stylesheet" href="/font-awesome/css/font-awesome.min.css">
  <link rel="apple-touch-icon" href="/apple-touch-icon.png">
  
    
    
      <meta name="google-site-verification" content="S3ijAtNZbVQ4VMOqNk3lqR3X9-GTumPoUokR6jgwkeE">
    
  
  
      <link rel="stylesheet" href="/fancybox/jquery.fancybox.css">
  
    
  <!-- 加载特效 -->
    <script src="/js/pace.js"></script>
    <link href="/css/pace/pace-theme-flash.css" rel="stylesheet">
  <script>
      var yiliaConfig = {
          rootUrl: '/',
          fancybox: true,
          animate: true,
          isHome: true,
          isPost: false,
          isArchive: false,
          isTag: false,
          isCategory: false,
          open_in_new: false
      }
  </script>
    <script src="https://cdn.bootcdn.net/ajax/libs/gitalk/1.6.2/gitalk.min.js"></script>
    <script>
        (function(){
            var bp = document.createElement('script');
            var curProtocol = window.location.protocol.split(':')[0];
            if (curProtocol === 'https') {
                bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
            }
            else {
                bp.src = 'http://push.zhanzhang.baidu.com/push.js';
            }
            var s = document.getElementsByTagName("script")[0];
            s.parentNode.insertBefore(bp, s);
        })();
    </script>
</head>
</html>
<body>
  <div id="container">
    <div class="left-col">
    <div class="overlay"></div>
<div class="intrude-less">
    <header id="header" class="inner">
        <a href="/" class="profilepic">
            
            <img lazy-src="/img/head.png" class="js-avatar">
            
        </a>

        <hgroup>
          <h1 class="header-author"><a href="/" title="Hi Mate">月 迹</a></h1>
        </hgroup>

        
        <p class="header-subtitle">用代码书写人生</p>
        
        
            <input type="text" class="search" id="search" style="text-align: center" placeholder="搜点什么...">
        
        
            <div id="switch-btn" class="switch-btn">
                <div class="icon">
                    <div class="icon-ctn">
                        <div class="icon-wrap icon-house" data-idx="0">
                            <div class="birdhouse"></div>
                            <div class="birdhouse_holes"></div>
                        </div>
                        <div class="icon-wrap icon-ribbon hide" data-idx="1">
                            <div class="ribbon"></div>
                        </div>
                        
                        <div class="icon-wrap icon-link hide" data-idx="2">
                            <div class="loopback_l"></div>
                            <div class="loopback_r"></div>
                        </div>
                        
                        
                        <div class="icon-wrap icon-me hide" data-idx="3">
                            <div class="user"></div>
                            <div class="shoulder"></div>
                        </div>
                        
                    </div>

                </div>
                <div class="tips-box hide">
                    <div class="tips-arrow"></div>
                    <ul class="tips-inner">
                        <li>菜单</li>
                        <li>标签</li>
                        
                        <li>友情链接</li>
                        
                        
                        <li>关于我</li>
                        
                    </ul>
                </div>
            </div>
        

        <div id="switch-area" class="switch-area">
            <div class="switch-wrap">
                <section class="switch-part switch-part1">
                    <nav class="header-menu">
                        <ul>
                        
                            <li><a href="index.html">博客首页</a></li>
                        
                            <li><a href="/archives">文章归档</a></li>
                        
                            <li><a href="/tags">文章统计</a></li>
                        
                            <li><a href="/message">交友留言</a></li>
                        
                            <li><a href="/navigation">资源导航</a></li>
                        
                            <li><a href="/about">关于作者</a></li>
                        
                        </ul>
                    </nav>
                    <nav class="header-nav">
                        <ul class="social">
                            
                                <a class="fl mail" target="_blank" href="http://mail.qq.com/cgi-bin/qm_share?t=qm_mailme&email=U2FhYmdrZGVjamITIiJ9MDw_" title="mail">mail</a>
                            
                                <a class="fl github" target="_blank" href="https://github.com/xyueji" title="github">github</a>
                            
                                <a class="fl rss" target="_blank" href="/atom.xml" title="rss">rss</a>
                            
                        </ul>
                    </nav>
                </section>

                
                <section class="switch-part switch-part2">
                    <div class="widget tagcloud" id="js-tagcloud">
                        <a href="tags/AppendTrie树/" style="font-size: 10px;">AppendTrie树</a> <a href="tags/HDFS/" style="font-size: 10px;">HDFS</a> <a href="tags/JanusGraph/" style="font-size: 10px;">JanusGraph</a> <a href="tags/SQL解析/" style="font-size: 10px;">SQL解析</a> <a href="tags/SQL采集/" style="font-size: 10px;">SQL采集</a> <a href="tags/alluxio/" style="font-size: 10px;">alluxio</a> <a href="tags/bitmap/" style="font-size: 10px;">bitmap</a> <a href="tags/container/" style="font-size: 10px;">container</a> <a href="tags/datasource/" style="font-size: 10px;">datasource</a> <a href="tags/docker/" style="font-size: 15px;">docker</a> <a href="tags/docker-compose/" style="font-size: 15px;">docker-compose</a> <a href="tags/flink/" style="font-size: 15px;">flink</a> <a href="tags/free/" style="font-size: 10px;">free</a> <a href="tags/git/" style="font-size: 10px;">git</a> <a href="tags/grafana/" style="font-size: 10px;">grafana</a> <a href="tags/hadoop/" style="font-size: 10px;">hadoop</a> <a href="tags/hbase/" style="font-size: 15px;">hbase</a> <a href="tags/hbck/" style="font-size: 10px;">hbck</a> <a href="tags/hive/" style="font-size: 15px;">hive</a> <a href="tags/idea/" style="font-size: 15px;">idea</a> <a href="tags/impala/" style="font-size: 10px;">impala</a> <a href="tags/iptables/" style="font-size: 10px;">iptables</a> <a href="tags/mac/" style="font-size: 10px;">mac</a> <a href="tags/mapreduce/" style="font-size: 10px;">mapreduce</a> <a href="tags/metrics2/" style="font-size: 10px;">metrics2</a> <a href="tags/opentsdb/" style="font-size: 10px;">opentsdb</a> <a href="tags/phoenix/" style="font-size: 15px;">phoenix</a> <a href="tags/presto/" style="font-size: 10px;">presto</a> <a href="tags/prometheus/" style="font-size: 10px;">prometheus</a> <a href="tags/pycharm/" style="font-size: 10px;">pycharm</a> <a href="tags/salt-api/" style="font-size: 10px;">salt-api</a> <a href="tags/shuffle/" style="font-size: 15px;">shuffle</a> <a href="tags/spark/" style="font-size: 15px;">spark</a> <a href="tags/springboot/" style="font-size: 10px;">springboot</a> <a href="tags/top/" style="font-size: 10px;">top</a> <a href="tags/vue/" style="font-size: 10px;">vue</a> <a href="tags/wordcount/" style="font-size: 10px;">wordcount</a> <a href="tags/修复工具/" style="font-size: 10px;">修复工具</a> <a href="tags/元数据/" style="font-size: 10px;">元数据</a> <a href="tags/分片/" style="font-size: 10px;">分片</a> <a href="tags/基础知识/" style="font-size: 10px;">基础知识</a> <a href="tags/大数据/" style="font-size: 20px;">大数据</a> <a href="tags/大文件/" style="font-size: 10px;">大文件</a> <a href="tags/大查询/" style="font-size: 10px;">大查询</a> <a href="tags/安装/" style="font-size: 10px;">安装</a> <a href="tags/小文件/" style="font-size: 10px;">小文件</a> <a href="tags/开发规范/" style="font-size: 10px;">开发规范</a> <a href="tags/数据倾斜/" style="font-size: 10px;">数据倾斜</a> <a href="tags/数据血缘/" style="font-size: 10px;">数据血缘</a> <a href="tags/测试/" style="font-size: 10px;">测试</a> <a href="tags/热门商品/" style="font-size: 10px;">热门商品</a> <a href="tags/端口/" style="font-size: 10px;">端口</a> <a href="tags/端口映射/" style="font-size: 10px;">端口映射</a> <a href="tags/精准去重/" style="font-size: 10px;">精准去重</a> <a href="tags/系统表/" style="font-size: 10px;">系统表</a> <a href="tags/表重建/" style="font-size: 10px;">表重建</a>
                    </div>
                </section>
                

                
                <section class="switch-part switch-part3">
                    <div id="js-friends">
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="https://dayesahh.github.io/">DaYesahh</a>
                    
                      <a target="_blank" class="main-nav-link switch-friends-link" href="http://www.54tianzhisheng.cn/">zhisjeng</a>
                    
                    </div>
                </section>
                

                
                
                <section class="switch-part switch-part4">
                
                    <div id="js-aboutme">爱动漫、追小说、喜影视、迷于严谨和逻辑的编码之中！！！</div>
                </section>
                
            </div>
        </div>
    </header>
</div>

    </div>
    <div class="mid-col">
      <nav id="mobile-nav">
      <div class="overlay">
          <div class="slider-trigger"></div>
          <h1 class="header-author js-mobile-header hide"><a href="/" title="Me">月 迹</a></h1>
      </div>
    <div class="intrude-less">
        <header id="header" class="inner">
            <a href="/" class="profilepic">
                
                    <img lazy-src="/img/head.png" class="js-avatar">
                
            </a>
            <hgroup>
              <h1 class="header-author"><a href="/" title="Me">月 迹</a></h1>
            </hgroup>
            
            <p class="header-subtitle">用代码书写人生</p>
            
            <nav class="header-menu">
                <ul>
                
                    <li><a href="index.html">博客首页</a></li>
                
                    <li><a href="/archives">文章归档</a></li>
                
                    <li><a href="/tags">文章统计</a></li>
                
                    <li><a href="/message">交友留言</a></li>
                
                    <li><a href="/navigation">资源导航</a></li>
                
                    <li><a href="/about">关于作者</a></li>
                
                <div class="clearfix"></div>
                </ul>
            </nav>
            <nav class="header-nav">
                <div class="social">
                    
                        <a class="mail" target="_blank" href="http://mail.qq.com/cgi-bin/qm_share?t=qm_mailme&email=U2FhYmdrZGVjamITIiJ9MDw_" title="mail">mail</a>
                    
                        <a class="github" target="_blank" href="https://github.com/xyueji" title="github">github</a>
                    
                        <a class="rss" target="_blank" href="/atom.xml" title="rss">rss</a>
                    
                </div>
            </nav>
        </header>                
    </div>
</nav>
      <div class="body-wrap">
  
    <article id="post-统计HDFS元数据" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="统计HDFS元数据.html" class="article-date">
      <time datetime="2021-09-23T09:17:45.000Z" itemprop="datePublished">2021-09-23</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="统计HDFS元数据.html">统计HDFS元数据</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <ol>
<li>编写offline_fsimage.sh脚本，解析fsimage文件获取HDFS元数据并上传至HDFS。</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 加载基本配置信息</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> <span class="built_in">source</span> ./config-env.sh</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 集群启用了Kerberos，先使用hdfs的Kerberos账号</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kinit -kt hdfs.keytab hdfs/admin</span></span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 将HDFS的FsImage导出</span></span><br><span class="line">mkdir -p ./tmp_meta</span><br><span class="line">rm -rf ./tmp_meta/*</span><br><span class="line">hdfs dfsadmin -fetchImage ./tmp_meta</span><br><span class="line"></span><br><span class="line">if [ $? -ne 0 ];then</span><br><span class="line">	echo "获取FsImage失败..."</span><br><span class="line">	exit</span><br><span class="line">fi</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 使用hdfs提供的oiv解析FsImage数据文件，将FsImage转换为csv格式数据</span></span><br><span class="line">hdfs oiv -i ./tmp_meta/* -o ./tmp_meta/fsimage.csv -p Delimited</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 将生成的csv文件头去掉，并上传至HDFS的/tmp目录</span></span><br><span class="line">sed -i -e "1d" ./tmp_meta/fsimage.csv</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 切换为Hive的用户进行put</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kinit -kt hive.keytab hive/admin</span></span><br><span class="line"></span><br><span class="line">TMP_DIR=/tmp/meta/fsimage</span><br><span class="line">hdfs dfs -rmr $TMP_DIR</span><br><span class="line">hdfs dfs -mkdir -p $TMP_DIR</span><br><span class="line">echo "创建HDFS临时目录[$&#123;TMP_DIR&#125;]"</span><br><span class="line">hdfs dfs -copyFromLocal ./tmp_meta/fsimage.csv $&#123;TMP_DIR&#125;</span><br><span class="line"></span><br><span class="line">if [ $? -ne 0 ];then</span><br><span class="line">	echo "上传fsimage.csv文件失败..."</span><br><span class="line">	exit</span><br><span class="line">fi</span><br><span class="line"></span><br><span class="line">echo "上传[./tmp_meta/fsimage.csv文件至[$&#123;TMP_DIR&#125;]成功"</span><br></pre></td></tr></table></figure>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/hadoop/">hadoop</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/HDFS/">HDFS</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/hive/">hive</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/元数据/">元数据</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/大查询/">大查询</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/小文件/">小文件</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="统计HDFS元数据.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-元数据平台-数据血缘" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="元数据平台-数据血缘.html" class="article-date">
      <time datetime="2021-09-03T09:39:59.000Z" itemprop="datePublished">2021-09-03</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="元数据平台-数据血缘.html">元数据平台-数据血缘</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <h3 id="架构"><a href="#架构" class="headerlink" title="架构"></a>架构</h3><p><img src="https://cdn.jsdelivr.net/gh/xyueji/img@master/web/image-20210903154320404.png" alt="image-20210903154320404"></p>
<p>首先，我们先了解一下元数据平台架构，主流程是：SQL采集 –》 SQL解析 –》应用层。</p>
<ol>
<li>SQL采集：针对各种SQL查询引擎，编写相应的钩子函数进行SQL收集，收集内容有执行时间、执行耗时、执行用户、执行引擎、jobId和执行SQL等等，最后把SQL信息入Kafka。Hive是实现ExecuteWithHookContext接口，Presto是实现EventListener接口，Spark是实现SparkListner接口。</li>
<li>SQL解析：Flink实时消费Kafka数据，进行SQL解析。解析SQL的过程为：定义词法规则和语法规则文件 –》 使用Antlr实现SQL词法和语法解析 –》生成AST语法树 –》遍历AST语法树，考虑到Presto和Spark的SQL语法类似，因此直接参考Hive底层源码实现SQL解析。解析完成后，把血缘信息和元数据信息分别入JanusGraph和ElasticSearch。</li>
</ol>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/元数据平台/">元数据平台</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/SQL解析/">SQL解析</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/SQL采集/">SQL采集</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/hive/">hive</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/大数据/">大数据</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/数据血缘/">数据血缘</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="元数据平台-数据血缘.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-精准去重-字典编码+Bitmap" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="精准去重-字典编码+Bitmap.html" class="article-date">
      <time datetime="2021-09-02T08:59:45.000Z" itemprop="datePublished">2021-09-02</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="精准去重-字典编码+Bitmap.html">精准去重-字典编码+Bitmap</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <p>Bitmap的基本思想是以一个bit位来表示一个元素对应的value，即使用bit数组下标来表示元素值，以大大缩小存储空间。BitMap一般用来快速查找、去重、删除等操作，但是它只能用于数字类型。那么如果要使用字符串类型的该怎么办呢？</p>
<p>这就需要先把字符串字典编码，生成字符串到数字的映射。本文参考kylin的全局字典编码配合RoaringBitmap以实现精准去重。</p>
<h3 id="AppendTrie树"><a href="#AppendTrie树" class="headerlink" title="AppendTrie树"></a>AppendTrie树</h3>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/功能小记/">功能小记</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/AppendTrie树/">AppendTrie树</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/bitmap/">bitmap</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/大数据/">大数据</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/精准去重/">精准去重</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="精准去重-字典编码+Bitmap.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-大数据监控-Metrics2录入指标到OpenTSDB" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="大数据监控-Metrics2录入指标到OpenTSDB.html" class="article-date">
      <time datetime="2021-08-09T06:10:59.000Z" itemprop="datePublished">2021-08-09</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="大数据监控-Metrics2录入指标到OpenTSDB.html">大数据监控-Metrics2录入指标到OpenTSDB</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <p>Hadoop Metrics2是一个收集各种指标的监控工具，可应用于Hadoop、HBase、Kylin等大数据组件监控指标收集。主要分为3个主要部分：MetricsSystemImpl、Source和Sink，其中Source和Sink都可以自定义。本文讲述如何自定义Sink，来收集指标入OpenTSDB。<br>首先来看看这3个部分的代码结构：</p>
<h4 id="MetricsSystemImpl"><a href="#MetricsSystemImpl" class="headerlink" title="MetricsSystemImpl"></a>MetricsSystemImpl</h4>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/集群监控/">集群监控</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/grafana/">grafana</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/hadoop/">hadoop</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/hbase/">hbase</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/metrics2/">metrics2</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/opentsdb/">opentsdb</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="大数据监控-Metrics2录入指标到OpenTSDB.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-Spark解决数据倾斜问题" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="Spark解决数据倾斜问题.html" class="article-date">
      <time datetime="2020-10-14T11:51:03.000Z" itemprop="datePublished">2020-10-14</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="Spark解决数据倾斜问题.html">Spark解决数据倾斜问题</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <p><strong>1、使用Hive ETL预处理数据方案适用场景：</strong><br>如果导致数据倾斜的是Hive表。如果该Hive表中的数据本身很不均匀（比如某个key对应了100万数据，其他key才对应了10条数据），而且业务场景需要频繁使用Spark对Hive表执行某个分析操作，那么比较适合使用这种技术方案。</p>
<blockquote>
<p>方案实现思路：</p>
</blockquote>
<p>此时可以评估一下，是否可以通过Hive来进行数据预处理（即通过Hive ETL预先对数据按照key进行聚合，或者是预先和其他表进行join），然后在Spark作业中针对的数据源就不是原来的Hive表了，而是预处理后的Hive表。此时由于数据已经预先进行过聚合或join操作了，那么在Spark作业中也就不需要使用原先的shuffle类算子执行这类操作了。</p>
<blockquote>
<p>方案实现原理：</p>
</blockquote>
<p>这种方案从根源上解决了数据倾斜，因为彻底避免了在Spark中执行shuffle类算子，那么肯定就不会有数据倾斜的问题了。但是这里也要提醒一下大家，这种方式属于治标不治本。因为毕竟数据本身就存在分布不均匀的问题，所以Hive ETL中进行group by或者join等shuffle操作时，还是会出现数据倾斜，导致Hive ETL的速度很慢。我们只是把数据倾斜的发生提前到了Hive ETL中，避免Spark程序发生数据倾斜而已。</p>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/spark/">spark</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/spark/">spark</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/数据倾斜/">数据倾斜</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="Spark解决数据倾斜问题.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-Spark-Shuffle原理" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="Spark-Shuffle原理.html" class="article-date">
      <time datetime="2020-10-14T11:49:19.000Z" itemprop="datePublished">2020-10-14</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="Spark-Shuffle原理.html">Spark-Shuffle原理</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <h2 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h2><p>Spark Shuffle是大众讨论的比较多的话题了。它是Spark任务执行过程中最为重要的过程之一。那么什么是Shuffle呢？<br>Shuffle一般被翻译成数据混洗，是类MapReduce分布式计算框架独有的机制，也是这类分布式计算框架最重要的执行机制。接下来会按照两个层面来谈谈Shuffle机制。分别为：</p>
<ul>
<li>逻辑层面</li>
<li>物理层面</li>
</ul>
<p>逻辑层面主要是从RDD的血缘出发，从DAG的角度来讲解Shuffle，另外也会说明Spark容错机制。物理层面是从执行角度来剖析Shuffle是如何发生的</p>
<h3 id="1-RDD血缘与Spark容错"><a href="#1-RDD血缘与Spark容错" class="headerlink" title="1. RDD血缘与Spark容错"></a>1. RDD血缘与Spark容错</h3>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/spark/">spark</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/shuffle/">shuffle</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/spark/">spark</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="Spark-Shuffle原理.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-MapReduce-Shuffle过程" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="MapReduce-Shuffle过程.html" class="article-date">
      <time datetime="2020-10-14T04:02:56.000Z" itemprop="datePublished">2020-10-14</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="MapReduce-Shuffle过程.html">MapReduce-Shuffle过程</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <blockquote>
<p>从map()的输出到reduce()的输入，中间的过程被称为shuffle过程。</p>
</blockquote>
<h2 id="map-side"><a href="#map-side" class="headerlink" title="map side"></a>map side</h2><ol>
<li><p>在写入磁盘之前，会先写入环形缓冲区(circular memory buffer)，默认100M(mapreduce.task.io.sort.mb可修改)，当缓冲区内容达到<br>80M(mapreduce.map.sort.spill.percent可修改)，缓冲区内容会被溢写到磁盘，形成一个spill file文件。</p>
</li>
<li><p>分区：在写入磁盘之前，会先进分区(partition)，而partition的数量是由reducer的数量决定的。</p>
<blockquote>
<p>job.setNumReduceTasks(2);</p>
</blockquote>
</li>
</ol>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/hadoop/">hadoop</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/mapreduce/">mapreduce</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/shuffle/">shuffle</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="MapReduce-Shuffle过程.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-Git开发规范" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="Git开发规范.html" class="article-date">
      <time datetime="2020-07-02T11:31:19.000Z" itemprop="datePublished">2020-07-02</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="Git开发规范.html">Git开发规范</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <h2 id="分支管理"><a href="#分支管理" class="headerlink" title="分支管理"></a>分支管理</h2><h3 id="分支命名"><a href="#分支命名" class="headerlink" title="分支命名"></a>分支命名</h3><p><strong>master 分支</strong></p>
<blockquote>
<ul>
<li>master 为主分支，也是用于部署生产环境的分支，确保master分支稳定性</li>
<li>master 分支一般由develop以及hotfix分支合并，任何时间都不能直接修改代码</li>
</ul>
</blockquote>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/git/">git</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/git/">git</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/开发规范/">开发规范</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="Git开发规范.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-Flink学习--如何计算实时热门商品" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="Flink学习--如何计算实时热门商品.html" class="article-date">
      <time datetime="2020-06-16T09:30:44.000Z" itemprop="datePublished">2020-06-16</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="Flink学习--如何计算实时热门商品.html">Flink学习--如何计算实时热门商品</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <p>首先我们把“实时热门商品”翻译成程序员易理解的语言：“每隔5分钟输出最近一小时内点击量最多的前N个商品”。将这个需求分解一下，将做如下几件事：</p>
<ul>
<li>构建数据源</li>
<li>抽取业务数据时间戳，告诉Flink基于业务时间做窗口</li>
<li>过滤出点击行为数据</li>
<li>按1小时的窗口，每隔5分钟，做滑动窗口聚合</li>
<li>按每个窗口聚合，输出每个窗口中点击量前N的商品</li>
</ul>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/flink/">flink</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/flink/">flink</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/大数据/">大数据</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/热门商品/">热门商品</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="Flink学习--如何计算实时热门商品.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
    <article id="post-Flink学习--基本概念与环境搭建" class="article article-type-post" itemscope itemprop="blogPost">
    
    <div class="article-meta">
      <a href="Flink学习--基本概念与环境搭建.html" class="article-date">
      <time datetime="2020-06-16T08:21:18.000Z" itemprop="datePublished">2020-06-16</time>
</a>
    </div>
  
  <div class="article-inner">
    
      <input type="hidden" class="isFancy" />
    
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="Flink学习--基本概念与环境搭建.html">Flink学习--基本概念与环境搭建</a>
    </h1>
  

      </header>
      
    
    <div class="article-entry" itemprop="articleBody">
      
          
              <h2 id="基本概念"><a href="#基本概念" class="headerlink" title="基本概念"></a>基本概念</h2><ul>
<li><p><a href="#数据流编程模型">Programming Model–数据流编程模型</a></p>
</li>
<li><p><a href="#分布式运行环境">Distributed Runtime–数据流编程模型</a></p>
</li>
</ul>
<h3 id="数据流编程模型"><a href="#数据流编程模型" class="headerlink" title="数据流编程模型"></a>数据流编程模型</h3>
          
      
      
    </div>
    
    <div class="article-info article-info-index">
      
      
    <div class="article-category tagcloud">
    <a class="article-category-link" href="categories/flink/">flink</a>
    </div>


      
    <div class="article-tag tagcloud">
        <ul class="article-tag-list"><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/flink/">flink</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/wordcount/">wordcount</a></li><li class="article-tag-list-item"><a class="article-tag-list-link" href="tags/大数据/">大数据</a></li></ul>
    </div>

      
        <p class="article-more-link">
          <a  href="Flink学习--基本概念与环境搭建.html#more">阅读全文 >></a>
        </p>
      
      <div class="clearfix"></div>
    </div>
    
  </div>
  
</article>











  
  
    <nav id="page-nav">
      <span class="page-number current">1</span><a class="page-number" href="page/2/">2</a><a class="page-number" href="page/3/">3</a><a class="extend next" rel="next" href="page/2/">Next &raquo;</a>
    </nav>
  
</div>
      <footer id="footer">
    <div class="outer">
        <div id="footer-info">
            <div class="footer-left">
                &copy; 2022 月 迹
            </div>
            <div class="footer-right">
                <a href="http://hexo.io/" target="_blank" style="color: #000">Hexo</a>  Theme <a href="https://github.com/luuman/hexo-theme-spfk" target="_blank" style="color: #000">spfk</a> by luuman
            </div>
        </div>
        
            <div class="visit">
                
                    <span id="busuanzi_container_site_pv" style='display:none'>
                        <span id="site-visit" >本站到访数:
                            <span id="busuanzi_value_site_uv"></span>
                        </span>
                    </span>
                
                
                    <span>, </span>
                
                
                    <span id="busuanzi_container_page_pv" style='display:none'>
                        <span id="page-visit">本页阅读量:
                            <span id="busuanzi_value_page_pv"></span>
                        </span>
                    </span>
                
            </div>
        
    </div>
</footer>

    </div>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.6/require.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
<script src="js/main.js"></script>

    <!--<script>-->
        <!--$(document).ready(function() {-->
            <!--var backgroundnum = 0;-->
            <!--var backgroundimg = "url(/background/bg-" + backgroundnum +".jpg)";-->
            <!--$("#mobile-nav").css({"background-image": backgroundimg,"background-size": "cover","background-position": "center"});-->
            <!--$(".left-col").css({"background-image": backgroundimg,"background-size": "cover","background-position": "center"});-->
        <!--})-->
    <!--</script>-->



<!-- Google Analytics -->
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-168200248-1"></script>
<script>
    window.dataLayer = window.dataLayer || [];
    function gtag(){dataLayer.push(arguments);}
    gtag('js', new Date());

    gtag('config', 'UA-168200248-1');
</script>

<!-- End Google Analytics -->



<script>
	var _hmt = _hmt || [];
	(function() {
		var hm = document.createElement("script");
		hm.src = "https://hm.baidu.com/hm.js?1c0d5d76b7455fc584663ce315f75837";
		var s = document.getElementsByTagName("script")[0];
		s.parentNode.insertBefore(hm, s);
	})();
</script>



<div class="scroll" id="scroll">
    <a href="#"><i class="fa fa-arrow-up"></i></a>
    <a href="#comments"><i class="fa fa-comments-o"></i></a>
    <a href="#footer"><i class="fa fa-arrow-down"></i></a>
</div>
<script>
    $(document).ready(function() {
        if ($("#comments").length < 1) {
            $("#scroll > a:nth-child(2)").hide();
        };
    })
</script>


<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    
    <script language="javascript">
    $(function() {
        $("a[title]").each(function() {
            var a = $(this);
            var title = a.attr('title');
            if (title == undefined || title == "") return;
            a.data('title', title).removeAttr('title').hover(

            function() {
                var offset = a.offset();
                $("<div id=\"anchortitlecontainer\"></div>").appendTo($("body")).html(title).css({
                    top: offset.top - a.outerHeight() - 15,
                    left: offset.left + a.outerWidth()/2 + 1
                }).fadeIn(function() {
                    var pop = $(this);
                    setTimeout(function() {
                        pop.remove();
                    }, pop.text().length * 800);
                });
            }, function() {
                $("#anchortitlecontainer").remove();
            });
        });
    });
</script>
    
    
    
        <div class="search-result-list" ></div>
        <div class="search-cover"></div>
    <script type = "text/javascript" >
        $('#search').bind('keypress',function(event){
            if(event.keyCode == "13"){
                search()
            }
        });

    function search() {
        var searchTool = {
            data: [],
            arrList: [],
            oldList: [],
            init() {
                this.fetchJson()
            },
            fetchJson() {
                window.fetch('/content.json?t=' + (+new Date()), {
                    method: 'get',
                }).then((res) => {
                    return res.json()
                }).then((data) => {
                    this.data = data
                    let keywords = $('#search').val()
                    this.searchList(keywords)
                }).catch((err) => {
                    console.log(err)
                });
            },
            // 搜索字符串里面是否存在关键字
            isSreachIndexOF(oldstr, kw) {
                // console.log(oldstr, kw)
                var istrue = false
                // console.log('isSreachIndexOF', oldstr && toString.call(oldstr) === '[object Array]')
                if (oldstr && toString.call(oldstr) === '[object Array]') {
                    for (var i = 0; i < oldstr.length; i++) {
                        oldstr[i].toLowerCase() === kw.toLowerCase() ? istrue = true : null
                    }
                    return istrue
                }
                if (!oldstr || !kw) return false
                istrue = oldstr.toLowerCase().indexOf(kw.toLowerCase()) > -1
                return istrue
            },
            searchList(keywolds) {
                let divList = ''
                if (!keywolds) {
                    if (this.oldList.length) this.clear()
                } else {
                    this.data.forEach((item, index) => {
                        let tagsK = item.tags.map(item => item.name).join('-')
                        let categoriesK = item.categories.map(item => item.name).join('-')
                        if (this.isSreachIndexOF(item.title, keywolds) || this.isSreachIndexOF(tagsK, keywolds) || this.isSreachIndexOF(categoriesK, keywolds) || this.isSreachIndexOF(item.content, keywolds)) {
                            divList += `
                          <li>
                            <a href="/${item.path}" class="search-result-title">${item.title}</a>
                            <div class="main">
                              <div class="left">
                                <span>${item.author ? item.author : 'xyueji'}</span>
                                ${item.tags.map(tags => `<a href="/tags/${tags.slug}" class="search-result-tags">${tags.name}</a>`)}
                              </div>
                              <div class="right">${this.getFriendlyTime(item.date.replace('T', ' '), new Date())}</div>
                            </div>
                          </li>`
                        }
                    })
                    if (this.oldList.length) {
                        this.clear()
                    }
                    if (this.arrList.length) this.add()
                }
                if (!divList) {
                    divList = `<a class="search_close" href="javascript:void(0)" onclick="searchToggle()">×</a> <div style="text-align: center;font-size: 30px; margin-top: 50px;" ">抱歉，未搜索到相关结果</div>`
                } else {
                    divList = `<a class="search_close" href="javascript:void(0)" onclick="searchToggle()">×</a> <ul>` + divList + `</ul>`
                }
                $('.search-result-list').html(divList)
                $('.search-result-list').fadeToggle();
                $('.search-cover').fadeToggle();
            },
            clear() {
                this.oldList.forEach(item => {
                    $(".search-result-list li")[item].style.display = 'none'
                })
                this.oldList = []
            },
            add() {
                this.arrList.forEach(item => {
                    $(".search-result-list li")[item].style.display = 'block'
                })
                this.oldList = this.arrList
            },
            searchGo(keywolds) {
                this.arrList = []
                if (!keywolds) {
                    if (this.oldList.length) this.clear()
                } else {
                    this.data.forEach((item, index) => {
                        if (this.isSreachIndexOF(item.title, keywolds) || this.isSreachIndexOF(item.tags.map(item => item.name).join('-'), keywolds) || this.isSreachIndexOF(item.categories.map(item => item.name).join('-'), keywolds) || this.isSreachIndexOF(item.content, keywolds)) {
                            this.arrList.push(index)
                        }
                    })
                    if (this.oldList.length) this.clear()
                    if (this.arrList.length) this.add()
                }
            },
            /**
             * 获取指定时间的友好时间字符串。
             * @param str 指定的时间字符串，如yyyy-MM-dd HH:mm:ss
             * @param now 当前时间，允许时间戳，GMT时间，如果该参数为undefined，则使用浏览器时间。
             */
            getFriendlyTime(str, now) {
                var currentTime = new Date(now);
                var arr = str.split(/\s+/gi);
                var temp = 0, arr1, arr2, oldTime, delta;
                var getIntValue = function (ss, defaultValue) {
                    try {
                        return parseInt(ss, 10);
                    } catch (e) {
                        return defaultValue;
                    }
                };
                var getWidthString = function (num) {
                    return num < 10 ? ('0' + num) : num;
                };
                if (arr.length >= 2) {
                    arr1 = arr[0].split(/[\/\-]/gi);
                    arr2 = arr[1].split(':');
                    oldTime = new Date();
                    oldTime.setYear(getIntValue(arr1[0], currentTime.getFullYear()));
                    oldTime.setMonth(getIntValue(arr1[1], currentTime.getMonth() + 1) - 1);
                    oldTime.setDate(getIntValue(arr1[2], currentTime.getDate()));

                    oldTime.setHours(getIntValue(arr2[0], currentTime.getHours()));
                    oldTime.setMinutes(getIntValue(arr2[1], currentTime.getMinutes()));
                    oldTime.setSeconds(getIntValue(arr2[2], currentTime.getSeconds()));

                    delta = currentTime.getTime() - oldTime.getTime();

                    if (delta <= 60 * 1000) {
                        return '1分钟内';
                    } else if (delta < 60 * 60 * 1000) {
                        return Math.floor(delta / (60 * 1000)) + '分钟前';
                    } else if (delta < 24 * 60 * 60 * 1000) {
                        return Math.floor(delta / (60 * 60 * 1000)) + '小时前';
                    } else if (delta < 24 * 60 * 60 * 1000 * 30) {
                        return Math.floor(delta / (24 * 60 * 60 * 1000)) + '天前';
                    } else if (delta < 24 * 60 * 60 * 1000 * 30 * 12) {
                        return Math.floor(delta / (24 * 60 * 60 * 1000 * 30)) + '月前';
                    } else if (currentTime.getFullYear() != oldTime.getFullYear()) {
                        return [getWidthString(oldTime.getFullYear()), getWidthString(oldTime.getMonth() + 1), getWidthString(oldTime.getDate())].join('-')
                    } else {
                        return [getWidthString(oldTime.getMonth() + 1), getWidthString(oldTime.getDate())].join('-');
                    }
                }
                return '';
            }
        }
        searchTool.init()
    }
    function searchToggle() {
        $('.search-cover').fadeToggle();
        $('.search-result-list').fadeToggle();
    }
</script>


  </div>
</body>
</html>
